namespace eval Data {
namespace export *
# read & write data file
proc read_data { fname data } {}
proc save {data { names {} } filename} {}
#define title 
proc read_title { fname } {}
proc get_title { base datafile } {}
proc attach_title { data title } {}

# operation
proc delete { item dlist } {}
proc filter { data condition } {}
proc add_col { data colname column } {}
proc calc {data operation} {}
# combining table
# cat data2 to data1 and return 
proc cat { data1 data2 } {}
# cat data2 into data1
proc dappend { data1 data2 } {}
# combine two datafiles have different column but same number of datapoints
proc combine { data1 data2} {}
# shortcuts: combine data files
proc combine_dfile {  datafile1 datafile2 datafile3 } {}
# shortcuts: combine data files found in the same subdir of base.
proc batch_combine { base datafile1 datafile2 datafile3 } {}
# shortcut: cat data files found in the subdirs of base
proc batch_cat2 { base datafile {condition  true } {optlist {} }} {}
proc collect {} {}
proc printTable {} {}
# statistic
#not defined group
proc group { data group col } {}
proc stat { data ave std } {}
proc prob { datacol bins } {}
proc searchbins { datapoint bins } {}
proc createbins { low high num } {}
proc ave {data col } {}
proc win_ave { data col bins } {}
#shortcut: 
proc win_ave_file {} {}

# helper

proc isinteger {v} {}
proc isfloat {v} {}
proc isnumber {v} {}
proc eachline {line data body } {}
proc reduce { olist } {}
proc format_title { title } {}
proc format_list { head flist formatstr } {}
proc max {list } {}
}

# read data from fname into array data
proc Data::read_data { fname data } {
    upvar $data Data
    set dfile [open $fname r ]
    set lines [read $dfile ]
    set lines [split $lines '\n']
    set ncol 0
    set numline 0
    set has_title false
    foreach line $lines {
	set dlist [regexp -inline -all -- {\S+} $line ] 
	if { [string first "##" $line ] == 0 } {
	    set has_title true 
            set title {}
	    foreach col [lrange $dlist 1 end] { 
             if {[string length $col] > 0} { lappend title $col}
            }
	    set ncol [llength $title ]
	    foreach t $title {
		set Data($t) [list]
	    }
	}
	if { [string first # $line ] == 0 } {
	    continue
	    # skip comment line
	}
	if { [expr $ncol == 0 ] && !$has_title } {
	    set ncol [llength $dlist ] 
	    set title [list]
	    for { set i 0 } { $i < $ncol } { incr i } {
		lappend title $i
	    }
	    foreach t $title {
		set Data($t) [list]
	    }
	}
	if { [llength $dlist ] == $ncol  }  {
	    for { set i 0 } { $i < $ncol } { incr i } {
		lappend Data([lindex $title $i]) [lindex $dlist $i ]
	    }
	    incr numline
	}
	
    }
    close $dfile
    return $numline
}
# save data into filename specify column names to include in the file.
proc Data::save {data { names {} } filename} {
    upvar $data Data
    set out [Path::open_file $filename]
    set str "## "
    if { [llength $names] ==0 } { set names [array names Data ] }
    set names [reduce $names]
    foreach name $names {
        append str "$name "
    }
    puts $out $str
    eachline line Data {
	set str ""
	array set dl $line
	foreach name $names {
	   append str "$dl($name)  "
	}
	puts $out $str
    }
   close $out
}
# save data into csv specify column names to include in the file.
proc Data::savecsv {data { names {} } filename} {
    upvar $data Data
    set out [Path::open_file $filename]
    if { [llength $names] ==0 } { set names [array names Data ] }
    set names [reduce $names]
    set str [join $names ,]
    puts $out $str
    eachline line Data {
        set str {}
        array set dl $line
        foreach name $names {
           lappend str $dl($name)
        }
        puts $out [join $str ,]
    }
   close $out
}

# get title from a directory base that contain data file 
proc Data::get_title { base datafile } {
    set Path $base 
    Path::search Path $datafile {
	set title [read_title $Path]
	if { [llength $title ] > 0 } { 
	    return $title
	}
    }
    return $title
}
# read title from file
proc Data::read_title { fname } {
    set dfile [open $fname r ]
    set lines [read $dfile ]
    close $dfile
    set lines [split $lines '\n']
    set ncol 0
    set numline 0
    set has_title false
    foreach line $lines {
	set dlist [regexp -inline -all -- {\S+} $line ] 
	if { [string first "##" $line ] == 0 } {
	    set has_title true 
	    set title [lrange $dlist 1 end]
	    set ncol [llength $title ]
	}
	if { $has_title } { return $title }
    } 
    if { ! $has_title } { return {}}
    return $title
}
proc Data::attach_title { data title } {
    upvar $data Data 
    set idx [lsort [array names Data] ]
    set ncol [llength $idx]
    for { set i 0 } { $i < $ncol } { incr i }  {
	set Data([lindex $title $i ]) $Data([lindex $idx $i])
	unset Data([lindex $idx $i])
    }
}


#operation



proc Data::filter { data condition } {
    upvar $data Data
    array set filtered {}
    set varlist {}
    foreach col [array names Data ] {
        set filtered($col) {}
	#determine which column is mentioned in the condition 
        if { [ string first $col $condition ] > -1} {
            lappend varlist $col
        }
    }
    if { [llength $varlist] == 0 } { error "condition incorrect!"; }
    set length [llength $Data([lindex $varlist 0])]
    set old $condition
    for { set i 0 } { $i < $length } { incr i } {
        set condition $old
        foreach var $varlist {
	    #replace @column name with its value
            regsub -all @$var $condition "[lindex $Data($var) $i ]" condition
        }
	#puts "[uplevel 1 "expr $condition"]"
        if { [uplevel 1 "expr $condition"  ] } {
            foreach col [array names Data] {
                lappend filtered($col) [lindex $Data($col) $i]
            }
        }
    }
    return [array get filtered]
}


proc Data::add_col { data colname column } {
    upvar $data Data
    set Data($colname) $column
}

proc Data::calc {data operation} {
    upvar $data Data
    set varlist {}
    foreach col [array names Data ] {
        set filtered($col) {}
	#find the variable names mentioned by the operation
        if { [ string first @$col $operation ] > -1} {
            lappend varlist $col
        }
    }
    set results {}
    set old $operation
    eachline line Data {
        set operation $old
        array set dataline $line
        foreach var $varlist {
            regsub -all @$var $operation $dataline($var) operation
        }
        set result [ uplevel 1 "expr $operation" ]
        lappend results $result
    }
    return $results
}
proc Data::sub {data1 data2 colname } {
   upvar $data1 Data1
   upvar $data2 Data2
   set result {}
   foreach col1 $Data1($col) col2 $Data2($col) {
    lappend result [expr $col1-$col2]
   }
   set Data1($col) $result
}
proc Data::add_row_num { data } {
   upvar $data Data
   set nrows [llength $Data([lindex [array names Data] 0])]
   set Data(RowNo) {}
   for {set i $start} {$i < [expr $start+$nrows] } {incr i }  {
   lappend Data(RowNo) $i
   }
   return $nrows
}
proc Data::cat { data1 data2 } {
    upvar $data1 Data1
    upvar $data2 Data2
    set cols1 [array names Data1]
    set cols2 [array names Data2]
    set cols {}
    array set data3 {}
    foreach col $cols1 {
        if { [lsearch $cols2 $col] >=0 }  {
            lappend cols $col
            set data3($col) {}
        }
    }
    foreach col $cols {
        set data3($col) [concat $Data1($col) $Data2($col)]
    }
    return [array get data3]
}

proc Data::dappend { data1 data2 } {
    upvar $data1 Data1
    upvar $data2 Data2
    set cols1 [array names Data1]
    set cols2 [array names Data2]
    set cols {}
    array set data3 {}
    if { [llength $cols1 ] == 0} {
        set cols1 $cols2
        foreach col $cols1 {
            set Data1($col) {}
        }
    }
    foreach col $cols1 {
        if { [lsearch $cols2 $col] >=0 }  {
            lappend cols $col
        }
    }
    foreach col $cols {
        set Data1($col) [concat $Data1($col) $Data2($col)]
    }

}
proc Data::combine { data1 data2} {
    upvar $data1 Data1
    upvar $data2 Data2
    set cols1 [array names Data1]
    set cols2 [array names Data2]
    set overlap {}
    set nlines [llength $Data1([lindex [array names Data1] 0]) ]
    array set data3 {}
    foreach col $cols1 {
        if { [llength $Data1($col)] != $nlines  } {
            error "Inconsisit lines in $data1!"}
        if ![info exists data3($col)] {
            set data3($col) $Data1($col)
        }
    }
    foreach col $cols2 {
        if { [llength $Data2($col)] != $nlines  } {
            error "Inconsisit lines in $data2!"}
        if ![info exists data3($col)] {
            set data3($col) $Data2($col)
        }
    }
    return [array get data3]
}


#shortcuts

proc Data::combine_dfile {  datafile1 datafile2 datafile3 } {
    array unset block1
    set nline [read_data $datafile1 block1]
    set title [read_title $datafile1 ]
    array unset block2
    set nline2 [read_data $datafile2 block2]
    set title [concat $title [read_title $datafile2 ]]
    array set block [combine block1 block2 ]
    save block $title $datafile3
}

proc Data::batch_combine { base datafile1 datafile2 datafile3 } {
    foreach path [Path::searchfiles $base $datafile1] {
        set path [file dirname $path ]
        puts "Under $path:\n combining $datafile1 & $datafile2 into $datafile3"
        if [catch { combine_dfile $path/$datafile1 $path/$datafile2 $path/$datafile3} error] {
             puts $error
        }
    }
}
proc Data::batch_cat { base datafile {pattern array(key)_value } args} {
    set idx [lsearch $args --save ] 
    set savepath $base
    if {$idx >= 0 } {
      set savepath [lindex $args [expr 1+$idx]] 
    }
    set idx [lsearch $args --combine ] 
    if { $idx >= 0 } { 
       set idx2 [lsearch -start [expr 1+$idx] $args --* ] 
       if { $idx2 < 0 } { set idx2 end}
       set combinecols [lrange $args [expr $idx+1] $idx2] 
    }
    set idx [lsearch $args --calc ]
    set calc_var {}
    if { $idx >= 0 } { 
       set idx2 [lsearch -start [expr 1+$idx] $args --* ] 
       if { $idx2 < 0 } { set idx2 end}
       set calc_var [lrange $args [expr $idx+1] $idx2] 
    }
    #find the possible values for variables
    if {[llength [set files [Path::searchfiles $base $datafile]]] <= 0} { error "No files $datafile found in $base!" }
    foreach f $files {
       if [catch { set vars [Path::parse $f $pattern] } error ] { puts $error; set vars {} } 
       foreach v $vars {
          upvar 0 $v V
          if ![Path::exists myspace $v] {
            Path::addVar myspace $v
          } else {
            Path::addOption myspace $v $V
          }
       }
    }
    #combine all columns
    set vars [Path::getVars myspace]
    if ![info exists combinecols] {set combinecols [Path::getVars myspace] }
    if {[lsearch $combinecols none] >= 0} { set combinecols {} }
    if {[lsearch $combinecols all ] >= 0} { set combinecols [Path::getVars myspace]}
    #add calc columns
    foreach col $calc_var {
        upvar 1 $col Col
        upvar 0 $col thisCol
        set thisCol $Col
        Path::addVar myspace $col
    }
    # generate conditins
    set cols {}
    foreach col $vars {
      if {[lsearch $combinecols $col] < 0 } { lappend cols $col }
    }
    set conditions {}
    set fnames {}
    while { [llength $cols] > 0 } {
         set col [lindex $cols 0]
         set cols [lrange $cols 1 end]
         set new_conditions {}
         set new_fnames {}
         foreach v [Path::getDef myspace $col] {
               if {[llength $conditions] == 0 } {
                   lappend new_conditions "\[string compare @$col $v \] == 0" 
                   lappend new_fnames "$col=$v" 
               } else {
               foreach c $conditions {
                   lappend new_conditions "$c && \[string compare @$col $v\] == 0"
               }
               foreach t $fnames {
                   lappend new_fnames "$t,$col=$v"
               }
              }
         } 
         set conditions $new_conditions
         set fnames $new_fnames
    }

    #puts "$conditions \n $fnames"
    #saving files
   if {[llength $conditions] <=0 } {
       set condition 1
       set result [Data::batch_cat2 $base $datafile $condition $pattern]
       set title [dict keys $result]
       array unset data
       array set data $result
       if {[llength [array names data]] > 0 } {
       puts "Saving $datafile into $savepath/$datafile ..."
       Data::save data $title $savepath/$datafile
       }
       set t {}
       foreach v $vars {
       lappend t $v
       lappend t [Path::getDef myspace $v]
       }
       return [concat $t [list fname $savepath/$datafile]]
   }
   for {set i 0} {$i <[llength $conditions] } { incr i } {
       set condition [lindex $conditions $i]
       set fname [lindex $fnames $i]
       set result [Data::batch_cat2 $base $datafile $condition $pattern]
       set title [dict keys $result]
       array unset data
       array set data $result
       if {[llength [array names data]] > 0 } {
       puts "saving $datafile into $savepath/$fname.$datafile"
       save data $title $savepath/$fname.$datafile
       }
   }
   #prepare for return
    set parsed {}
    foreach f $fnames {
      set t [string map { = " " } $f]
      set t [string map { , " " } $t]
      foreach {var value } $t {
        set myspace($var) $value
      }
      foreach var $calc_var {
      upvar 1 $var Var
      set result [Path::evaluate myspace $var $Var]
      lappend t $var
      lappend t $result
      } 
      lappend parsed [concat $t [list fname $savepath/$f.$datafile]]
    }
    return $parsed
}

proc Data::batch_cat2 { base datafile {condition  1 } {pattern array(key)_value }} {
    set Path $base
    array set Data {}
    set old $condition
    #puts $condition
    Path::search Path $datafile {
        set path [file dirname $Path]
        set vlist [ Path::parse $Path $pattern ]
        set title [ Data::read_title $Path]
        set condition $old
        foreach v $vlist {
          upvar 0 $v V
          set condition [string map [list @$v $V] $condition]
        }
        if {[uplevel 1 " expr $condition" ] } {
            array unset block
      #     puts $path
            read_data $Path block
     #       parray block
            dappend Data block
     #       parray Data
        }
       #puts $path
       #puts $title
       #puts [array names Data]
    }
    foreach col $title {
       if [info exists Data($col)] { dict set result $col $Data($col) } else { set result {} }
    }
    return $result
}
proc Data::batch_cat3 { base datafiles } {
    array set Data {}
    foreach dfile $datafiles {
       set title [Data::read_title $dfile]
       array unset block
       read_data $dfile block
       dappend Data block
    }
    foreach col $title {
       if [info exists Data($col)] { dict set result $col $Data($col) } else { set result {} }
    }
    return $result   
}


# collect data from datafile using a function called prossessor
# return a dictionary
proc Data::collect { path datafile pattern processor} {
    set entropy_files [Path::searchfiles $path $datafile]
    set S [dict create]
    foreach f $entropy_files {
        set vars [Path::parse $f $pattern]
        set key {}
        foreach v $vars {
            upvar 0 $v V
            if ![info exists Values($v)] { set Values($v) $V } else {
                if {[lsearch $Values($v) $V] < 0 } { lappend Values($v) $V }
            }
            lappend key  $v
            lappend key $V
        }
        set result [ eval $processor $f]
        dict lappend S $key $result
    }
    return $S
}
#convert collected data dictionary to table
proc Data::toTable { dict } {
    set key_var [getDictKeys [lindex [dict keys $dict] 0]]
    set key_value [getDictKeys [lindex [dict get $dict [lindex [dict keys $dict] 0] ] 0]]
    set nkey [llength $key_value]
    set str "##[join  $key_var " " ]"
    append str "  [join $key_value " " ]\n"
    foreach {key value } $dict {
            foreach v $value {
                append str " [getDictValues $key] "
                append str " [getDictValues $v] \n"
            }
    }
   return $str
}



#
proc Data::stat { data ave std } {
    upvar $data Data
    upvar $ave Ave
    upvar $std Std
    foreach name [array names Data] {
        set Ave($name) 0
        set Std($name) 0
        set Num($name) 0
        foreach v $Data($name) {
            set Ave($name) [expr $Ave($name) +$v ]
            set Std($name) [expr $Std($name) +$v*$v ]
            set Num($name) [expr $Num($name) +1 ]
        }
        if {$Num($name) <= 0} {set Ave($name) 0; set Std($name) 0 } else {
        if [catch {
        set Ave($name) [expr $Ave($name)/$Num($name) ]
        set Std($name) [expr sqrt($Std($name)/$Num($name)-$Ave($name)*$Ave($name))]
          }] { puts "$Ave($name) $Std($name)";set Ave($name) NA; set Std($name) NA;}
        }
    }
}
proc Data::stat2 {  data ave std } {
    upvar $data Data
    upvar $ave Ave
    upvar $std Std
    if [info exists Data(title)] {
        set cols $title
    } else {
        set cols [array names Data]
    }
    eachline line Data {
        array set dataline $line
        foreach col $cols {
            set Ave($col)
        }
    }
}

proc Data::prob { datacol bins } {
    array set Count {}
    foreach datapoint $datacol {
        set idx [searchbins $datapoint $bins]
        if { $idx == -1} { set x -Inf }
        if { $idx == -2 } { set x +Inf }
        if { $idx >= 0 } { set x [lindex $bins $idx] }
        if { ![info exists Count($x) ] } {
            set Count($x) 1
        } else {
            incr Count($x)
        }
    }
    return [array get Count]
}
proc Data::win_ave { data col bins ave std num} {
    upvar $data Data
    upvar $ave Ave
    upvar $num Num
    upvar $std Std
    array set Ave {}
    array set Num {}
    array set Std {}
    eachline line Data {
        array set dataline $line
        set idx [searchbins $dataline($col) $bins]
           if ![info exist Num($idx)] {
              set Num($idx) 1
           } else {
              set Num($idx) [expr $Num($idx) +1]
           }
        foreach name [array names dataline ] {
           if { $name == $col } { continue }
           if ![info exist Ave($name,$col=$idx)] {
              set Ave($name,$col=$idx) $dataline($name)
              set Std($name,$col=$idx) [expr $dataline($name)*$dataline($name)]
        } else {
              set Ave($name,$col=$idx) [expr $Ave($name,$col=$idx) +$dataline($name) ]
              set Std($name,$col=$idx) [expr $Std($name,$col=$idx) +$dataline($name)*$dataline($name)]
       }
    }
  }
   foreach name [array names Data] {
     if { $name == $col } { continue }
     for { set idx -2} { $idx < [llength $bins] } { incr idx } {
        if {[info exists Num($idx)] && $Num($idx) > 0  }  {
        set Ave($name,$col=$idx) [expr $Ave($name,$col=$idx) /$Num($idx)]
        set Std($name,$col=$idx) [expr $Std($name,$col=$idx)/$Num($idx) -$Ave($name,$col=$idx)*$Ave($name,$col=$idx)]
        if { $Num($idx) > 1 } {
        set Std($name,$col=$idx) [expr sqrt($Std($name,$col=$idx))]
         } else {
        set Std($name,$col=$idx) 0.0
         }
        } else {
        set Num($idx) 0
        set Std($name,$col=$idx) NA
        set Ave($name,$col=$idx) 0
        }
     }
   }
   return [array get Ave]
}

proc Data::win_ave_file { file col binlow binhigh nbin names resultfile} {
read_data $file data
set names [read_title $file]
delete $col names
set bins [createbins $binlow $binhigh $nbin]
win_ave data $col $bins ave std num
set result($col)  [list "<[lindex $bins 0]" ]
set result(Count($col)) [list $num(-1)]
set title [concat $col Count($col) $names ]
foreach name $names {
set result($name) [list $ave($name,$col=-1)]
}
foreach idx [lsort -increasing -real [array names num]] {
  if { $idx >= 0 }  {
  lappend result($col) [lindex $bins $idx]
  lappend result(Count($col)) $num($idx)
  foreach name $names {
  lappend result($name) $ave($name,$col=$idx)
  }
  }
}
lappend result($col) [list ">[lindex $bins end]" ]
lappend result(Count($col)) $num(-2)
foreach name $names {
lappend result($name)  $ave($name,$col=-2)
}

save result $title $resultfile
}

proc Data::createbins { low high num } {
    set delta [expr (double($high)-double($low))/$num]
    set bins {}
    for { set i 0} { $i <= $num } { incr i } {
        lappend bins [expr double($low)+$i*$delta]
    }
    return $bins
}

#helper find which bin datapoint falls in
proc Data::searchbins { datapoint bins } {
    set lowidx 0
    set highidx [expr [llength $bins]-1]
    set mididx [expr ($lowidx+$highidx)/2]
    set low [lindex $bins $lowidx]
    set high [lindex $bins $highidx]
    set mid [lindex $bins $mididx ]
    if {$datapoint < $low } {return -1 }
    if {$datapoint > $high } {return -2 }
    while { [expr $highidx-$lowidx] >1 } {
        if { $datapoint < $mid } {
            set high $mid
            set highidx $mididx
            set mididx [expr ($highidx+$lowidx)/2]
            set mid [lindex $bins $mididx]
        } elseif { $datapoint > $mid } {
            set low $mid
            set lowidx $mididx
            set mididx [expr ($highidx+$lowidx)/2]
            set mid [lindex $bins $mididx]
        } else {
            set lowidx $mididx
            set highidx $mididx
            set low $mid
            set high $mid
        }
    }
    return $mididx
}



#helper
# do sth for eachline 
proc Data::eachline {line data body } {
    upvar $data Data
    upvar $line Line
    if {![info exists Line]} { set Line {} }
    set length [llength $Data([lindex [array names Data] 0])]
    for { set i 0 } { $i <$length } {incr i } {
        set Line {}
        foreach name [array names Data] {
            set value [lindex $Data($name) $i]
            lappend Line $name
            lappend Line $value
        }
#       array set l $Line
        uplevel $body
    }
}

proc Data::delete { item dlist } {
    upvar $dlist Dlist
    set idx [lsearch $Dlist $item]
    set Dlist [lreplace $Dlist $idx $idx]
}
# remove duplicated elements from list
proc Data::reduce { olist } {
     set result {}
     foreach l $olist {
        set idx [lsearch $result $l]
        if { $idx < 0 } {
        lappend result $l
        }
     }
   return $result
}

# make title 5 character wide per column
proc Data::format_title { title } {
    set str ""
    foreach t $title {
	append str [format "%5s " $t ]
    }
    return $str
}

proc Data::format_list { head flist formatstr } {
    set str [format "%5s " $head]
    foreach t $flist {
	append str [format $formatstr $t ]
    }
    return $str
}

proc Data::isinteger {v} {
    return [regexp {^[+-]?[0-9]+$} $v]
}

proc Data::isfloat {v} {
    return [regexp {^[+-]?(([0-9]*\.[0-9]+)|([0-9]+\.[0-9]*)|([0-9]+))((e|E)[+-]?[0-9]+)?$} $v]
}

proc Data::isnumber {v} {
    return [expr [Data::isinteger $v] || [Data::isfloat $v ]]
}

proc Data::max { datalist } {
    set max [lindex $datalist 0]
    foreach data $datalist {
	if {$data > $max } {set max $data} 
    }
    return $max
}
proc Data::min { datalist } {
    set min [lindex $datalist 0]
    foreach data $datalist {
	if {$data < $min } {set min $data }
    }
    return $min
}
proc Data::len { datalist } {
   return [llength $datalist ]
}
proc Data::sum {datalist } {
   set sum 0.0
   set n [llength $datalist]
   if {$n <=0 } { return NA }
   foreach v $datalist {
      set sum [expr $sum +$v]
   }
   return $sum
}
proc Data::ave {datalist } {
   set sum 0.0
   set n [llength $datalist]
   if {$n <=0 } { return NA }
   foreach v $datalist {
      set sum [expr $sum +$v]
   }
   return [expr $sum/$n]
}
proc Data::std {datalist} {
   set sum 0.0
   set n [llength $datalist]
   if {$n <=1 } { return NA }
   foreach v $datalist {
      set sum [expr $sum +$v*$v]
   }
   set ave [Data::ave $datalist]
   return [expr sqrt( $sum/$n-$ave*$ave)]
}
proc Data::getDictValues {dict } {
    set result {}
    for {set i 1} { $i < [llength $dict]} {incr i 2 } {
        lappend result [lindex $dict $i]
    }
    return $result
}
proc Data::getDictKeys {dict } {
    set result {}
    for {set i 0} { $i < [llength $dict]} {incr i 2 } {
        lappend result [lindex $dict $i]
    }
    return $result
}
proc Data::lremove { list toremove } {
   set result {}
   foreach item $list {
    if {[lsearch $toremove $item] <0} { lappend result $item }
   }
   return $result
}

proc Data::ladd { list values } {
    upvar $list L
    if ![info exists L] { set L {} }
    foreach value $values {
        if { [lsearch $L $value ] < 0 } { uplevel 1 "lappend $list $value" }
    }
}


proc Data::addRowNo { data {start 1 } } {
           upvar $data Data
                      set nrows [llength $Data([lindex [array names Data] 0])]
                         set Data(RowNo) {}
              for {set i $start} {$i < [expr $start+$nrows] } {incr i }  {
                         lappend Data(RowNo) $i
                                    }
                 return $nrows
}


proc Data::group { data columnlist} {
          upvar $data Data
          array set groups {}
          set i 0
          eachline line Data {
         # set key ""
            set key [dict create]
            foreach col $columnlist {
                #append key $col=[dict get $line $col],
                dict set key $col [dict get $line $col]
            }
            if [info exists groups($key)] { 
                lappend groups($key) $i
            } else {
               set groups($key) [list $i]
            }
            incr i
        }
        return [array get groups]
}

proc Data::subtotal { data group operation } {
     upvar $data Data
     array set Groups $group
     foreach col [array names Data] {
        set Subtotal($col) {}
     }
     foreach key [lsort [array names Groups]] {
       set lineNo $Groups($key)
       foreach col [array names Data] {
           set d {}
           foreach l $lineNo {lappend d [lindex $Data($col) $l] }
           if [catch {
               set result [uplevel 0 $operation [list $d ]]}] {
               set result "\"[join [Data::reduce $d] ,]\" "
           }
           if [dict exists $key $col ]  {
               lappend Subtotal($col) [dict get $key $col]
           } else {
               lappend Subtotal($col) $result
           }
       }
     }
    return [array get Subtotal]
}
